/**
 * @author Fang Lu, fanglupku@gmail.com
 * 2011-5-14
 */
package com.autotag.nlp;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;

import com.autotag.FunctionalityTagGenerator;
import com.autotag.WSDLStopWord;
import com.autotag.entity.NPEntity;
import com.autotag.entity.TagEntity;

import fanglu.util.FileDealer;

public class GramStat {
	public static void main(String args[]){
//		File stemMapUUIDFile = new File("C:\\Users\\ftc\\Desktop\\ICSOC\\WSDL_FILE\\result\\stem_uuid.txt");
//		File UUIDMapStemFile = new File("C:\\Users\\ftc\\Desktop\\ICSOC\\WSDL_FILE\\result\\uuid_stem.txt");
//		HashMap<String, HashMap<String, Integer>> stemMapUUID = initGramFromMapFile(stemMapUUIDFile);
//		HashMap<String, HashMap<String, Integer>> UUIDMapStem = initGramFromMapFile(UUIDMapStemFile);
//		System.out.println(stemMapUUID.keySet().size());
//		System.out.println(UUIDMapStem.keySet().size());
		
//		HashMap<String, HashMap<String, Integer>> stemMapUUID = new HashMap<String, HashMap<String,Integer>>();
//		HashMap<String, HashMap<String, Integer>> UUIDMapStem = new HashMap<String, HashMap<String,Integer>>();
//		initGramFromInfoFile("C:\\Users\\ftc\\Desktop\\ICSOC\\WSDL_FILE\\result\\result.txt", 1, stemMapUUID, UUIDMapStem);
		
//		generateGramData();
		
//		generateStemStat();
		
//		generateGramData();
		
		staticTerm("C:\\Users\\ftc\\Desktop\\ICSOC\\WSDL_FILE\\result\\result.txt", 
				"C:\\Users\\ftc\\Desktop\\ICSOC\\WSDL_FILE\\result\\term.txt", 5);
	}
	public static void initTermMap(HashMap<String, Integer> termMapID, HashMap<Integer, String> IDMapTerm, 
			String filename){
		try {
			String content = FileDealer.getFileContent(filename);
			BufferedReader br = new BufferedReader(new StringReader(content));
			String oneLine  = br.readLine();
			while (oneLine != null){
				String[] sa = oneLine.trim().split(",");
				if (sa.length == 2){
					String thisTerm = sa[0];
					try{
						Integer thisID = Integer.parseInt(sa[1]);
						termMapID.put(thisTerm, thisID);
						IDMapTerm.put(thisID, thisTerm);
					}catch (Exception e) {
						// TODO: handle exception
					}
				}
				oneLine = br.readLine();
			}
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		
	}
	public static void staticTerm(String infoFileName, String termFileName, int level_sum){
		HashMap<String, Integer> termMapID = new HashMap<String, Integer>();
		try {
			File infoFile = new File(infoFileName);
			FileInputStream fis = new FileInputStream(infoFile);
			InputStreamReader isr = new InputStreamReader(fis);
			BufferedReader br = new BufferedReader(isr);
			String oneLine = br.readLine();
			int term_index = 1;
			try{
				while(oneLine != null){
					if (oneLine.startsWith("#") 
							&& oneLine.endsWith(".html")){
					}else{
						String [] strArray = oneLine.split("\\" + FunctionalityTagGenerator.CONTENT_SEPERATOR);
						for (int i=0; i<strArray.length; ++i){
							String [] a2 = strArray[i].split("\\s");
							for (int j=0; j<a2.length; ++j){
								String thisToken = WordStemmer.getStem(a2[j].trim());
								if (thisToken.length() > 1
										&& !WSDLStopWord.isStopWord(thisToken)
										&& WordStemmer.rootMap.containsKey(thisToken)){
									if (!termMapID.containsKey(thisToken)){
										termMapID.put(thisToken, term_index);
										++ term_index;
									}
								}
							}
							
						}
						
					}
							
					oneLine = br.readLine();
				}
			}catch (Exception e) {
				e.printStackTrace();
			}
			
			
			br.close();
			isr.close();
			fis.close();
			
			File file = new File(termFileName);
			BufferedWriter bw = new BufferedWriter(new FileWriter(file));
			Iterator<String> itor = termMapID.keySet().iterator();
			while(itor.hasNext()){
				String thisTerm = itor.next();
				Integer thisID = termMapID.get(thisTerm);
				bw.write(thisTerm + "," + thisID + "\n");
			}
			bw.close();
		} catch (FileNotFoundException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		
		
	}
	
	public static void generateStemStat(){
		HashMap<String, HashMap<String, Integer>> stemMapUUID = new HashMap<String, HashMap<String,Integer>>();
		HashMap<String, HashMap<String, Integer>> UUIDMapStem = new HashMap<String, HashMap<String,Integer>>();
		HashMap<String, HashMap<String, HashSet<Integer>>> UUIDMapStemLevel = new HashMap<String, HashMap<String,HashSet<Integer>>>();
		HashMap<String, HashMap<String, NPEntity>> UUIDMapNP = new HashMap<String, HashMap<String,NPEntity>>();
		initGramFromInfoFile("C:\\Users\\ftc\\Desktop\\ICSOC\\WSDL_FILE\\result\\result.txt", 1,
				5, stemMapUUID, UUIDMapStem, UUIDMapStemLevel, UUIDMapNP);
		ArrayList<TagEntity> tagList_DF = new ArrayList<TagEntity>();
		ArrayList<TagEntity> tagList_Total = new ArrayList<TagEntity>();
		Iterator<String> itor = stemMapUUID.keySet().iterator();
		while (itor.hasNext()){
			String thisStem = itor.next();
			TagEntity tag_DF = new TagEntity(thisStem, 1, stemMapUUID.get(thisStem).size());
			tagList_DF.add(tag_DF);
			HashMap<String, Integer> UUIDMapCount = stemMapUUID.get(thisStem);
			Iterator<String> itor_sub = UUIDMapCount.keySet().iterator();
			int count = 0;
			while (itor_sub.hasNext()){
				String thisUUID = itor_sub.next();
				count += UUIDMapCount.get(thisUUID);
			}
			TagEntity tag_Total = new TagEntity(thisStem, 1, count);
			tagList_Total.add(tag_Total);
		}
		Collections.sort(tagList_DF);
		File file1 = new File("C:\\Users\\ftc\\Desktop\\ICSOC\\WSDL_FILE\\result\\stem_stat_df.txt");
		try {
			BufferedWriter bw = new BufferedWriter(new FileWriter(file1));
			
			for (int i=0; i<tagList_DF.size(); ++i){
				bw.write(i + "," + tagList_DF.get(i).getName() + 
						"," + tagList_DF.get(i).getWeight() + "\n");
			}
			
			bw.close();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		Collections.sort(tagList_Total);
		File file2 = new File("C:\\Users\\ftc\\Desktop\\ICSOC\\WSDL_FILE\\result\\stem_stat_total.txt");
		try {
			BufferedWriter bw = new BufferedWriter(new FileWriter(file2));
			
			for (int i=0; i<tagList_Total.size(); ++i){
				bw.write(i + "," + tagList_Total.get(i).getName() + 
						"," + tagList_Total.get(i).getWeight() + "\n");
			}
			
			bw.close();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}
	
	@SuppressWarnings("unchecked")
	public static HashMap<String, HashMap<String, Integer>> initGramFromMapFile(File file){
		
		try {
			ObjectInputStream r1 = new ObjectInputStream(new FileInputStream(file));
			HashMap<String, HashMap<String, Integer>> targetMap = (HashMap<String, HashMap<String, Integer>>) r1.readObject();
			r1.close();
			return targetMap;
		} catch (FileNotFoundException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (ClassNotFoundException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		return null;
		
	}
	public static void generateGramData(){
		HashMap<String, HashMap<String, Integer>> stemMapUUID = new HashMap<String, HashMap<String,Integer>>();
		HashMap<String, HashMap<String, Integer>> UUIDMapStem = new HashMap<String, HashMap<String,Integer>>();
		HashMap<String, HashMap<String, HashSet<Integer>>> UUIDMapStemLevel = new HashMap<String, HashMap<String,HashSet<Integer>>>();
		HashMap<String, HashMap<String, NPEntity>> UUIDMapNP = new HashMap<String, HashMap<String,NPEntity>>();
		initGramFromInfoFile("C:\\Users\\ftc\\Desktop\\ICSOC\\WSDL_FILE\\result\\result.txt", 1,
				5, stemMapUUID, UUIDMapStem, UUIDMapStemLevel, UUIDMapNP);
		System.out.println(stemMapUUID.size());
		System.out.println(UUIDMapStem.size());
		System.out.println(UUIDMapStemLevel.size());
		System.out.println(UUIDMapNP.size());
		try{
			File file1 = new File("C:\\Users\\ftc\\Desktop\\ICSOC\\WSDL_FILE\\result\\stem_uuid.txt");
			File file2 = new File("C:\\Users\\ftc\\Desktop\\ICSOC\\WSDL_FILE\\result\\uuid_stem.txt");
			File file3 = new File("C:\\Users\\ftc\\Desktop\\ICSOC\\WSDL_FILE\\result\\uuid_stemlevel.txt");
			File file4 = new File("C:\\Users\\ftc\\Desktop\\ICSOC\\WSDL_FILE\\result\\uuid_np.txt");
			saveObjectToFile(stemMapUUID, file1.getAbsolutePath());
			saveObjectToFile(UUIDMapStem, file2.getAbsolutePath());
			saveObjectToFile(UUIDMapStemLevel, file3.getAbsolutePath());
			saveObjectToFile(UUIDMapNP, file4.getAbsolutePath());
			
		}catch(Exception e){
				
		}
	}
	public static Object readObjectFromFile(String fileName){
		Object result = null;
		try {
			ObjectInputStream r1 = new ObjectInputStream(new FileInputStream(fileName));
			result = r1.readObject();
			r1.close();
		} catch (FileNotFoundException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (ClassNotFoundException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		return result;
	}
	public static void saveObjectToFile(Object object, String fileName){
		File file = new File(fileName);
		try {
			ObjectOutputStream w1 = new ObjectOutputStream(new FileOutputStream(file));
			w1.writeObject(object);
			w1.close();
		} catch (FileNotFoundException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}
	public static void initGramFromInfoFile(String infoFileName, int max_n, int level_sum,
			HashMap<String, HashMap<String, Integer>> stemMapUUID, 
			HashMap<String, HashMap<String, Integer>> UUIDMapStem,
			HashMap<String, HashMap<String, HashSet<Integer>>> UUIDMapStemLevel,
			HashMap<String, HashMap<String, NPEntity>> UUIDMapNP){
		
		try {
			int WSDL_count = 0;
			File infoFile = new File(infoFileName);
			FileInputStream fis = new FileInputStream(infoFile);
			InputStreamReader isr = new InputStreamReader(fis);
			BufferedReader br = new BufferedReader(isr);
			String oneLine = br.readLine();
			try{
				while(oneLine != null){
					if (oneLine.startsWith("#") 
							&& oneLine.endsWith(".html")){
//						System.out.println(WSDL_count);
						++ WSDL_count;
						String thisUUID = new String(oneLine.substring(1, oneLine.length()-5));
						HashMap<String, Integer> thisStemMapCount = new HashMap<String, Integer>();
						UUIDMapStem.put(thisUUID, thisStemMapCount);
						HashMap<String, HashSet<Integer>> thisStemMapLevel = new HashMap<String, HashSet<Integer>>();
						UUIDMapStemLevel.put(thisUUID, thisStemMapLevel);
						HashMap<String, NPEntity> thisNPMap = new HashMap<String, NPEntity>();
						UUIDMapNP.put(thisUUID, thisNPMap);
						for (int i=0; i<level_sum; ++i){
							oneLine = br.readLine();
							if (oneLine.startsWith("#")){
								thisUUID = new String(oneLine.substring(1, oneLine.length()-5));
								thisStemMapCount = new HashMap<String, Integer>();
								UUIDMapStem.put(thisUUID, thisStemMapCount);
								thisStemMapLevel = new HashMap<String, HashSet<Integer>>();
								UUIDMapStemLevel.put(thisUUID, thisStemMapLevel);
								thisNPMap = new HashMap<String, NPEntity>();
								UUIDMapNP.put(thisUUID, thisNPMap);
								--i;
								continue;
							}
							String thisContent = new String(oneLine.substring(2));
							String [] strArray = thisContent.split("\\" + FunctionalityTagGenerator.CONTENT_SEPERATOR);
							for (int j=0; j<strArray.length; ++j){
								String thisStr = strArray[j];
								//1. deal with Noun Phrase
//								if (i < 2){
//									String taggedString = AnalyzeResultOfTagging.taggerString(thisStr.toLowerCase());
////									System.out.println("s");
//									ArrayList<String> NPList = NPChunkDealer.getNounPhraseList(taggedString, true);
////									System.out.println("e");
//									for (String thisNPString : NPList) {
//										NPEntity thisNPEntity = thisNPMap.get(thisNPString);
//										if (thisNPEntity == null){
//											HashMap<Integer, Integer> levelMapCount = new HashMap<Integer, Integer>();
//											levelMapCount.put(i+1, 1);
//											thisNPEntity = new NPEntity(thisNPString, levelMapCount);
//											thisNPMap.put(thisNPString, thisNPEntity);
//											if (thisNPEntity.getTokenNum() > 1){
//												HashMap<String, Integer> thisUUIDMapCount = stemMapUUID.get(thisNPString);
//												if (thisUUIDMapCount == null){
//													thisUUIDMapCount = new HashMap<String, Integer>();
//													stemMapUUID.put(thisNPString, thisUUIDMapCount);
//												}
//												Integer thisCount = thisUUIDMapCount.get(thisUUID);
//												if (thisCount == null){
//													thisUUIDMapCount.put(thisUUID, 1);
//												}else{
//													thisUUIDMapCount.put(thisUUID, thisCount + 1);
//												}
//												thisCount = thisStemMapCount.get(thisNPString);
//												if (thisCount == null){
//													thisStemMapCount.put(thisNPString, 1);
//												}else{
//													thisStemMapCount.put(thisNPString, thisCount + 1);
//												}
//												HashSet<Integer> thisSet = thisStemMapLevel.get(thisNPString);
//												if (thisSet == null){
//													thisSet = new HashSet<Integer>();
//													thisStemMapLevel.put(thisNPString, thisSet);
//												}
//												thisSet.add(i+1);
//											}
//										}else{
//											HashMap<Integer, Integer> levelMapCount = thisNPEntity.getLevelMapCount();
//											Integer thisCount = levelMapCount.get(i+1);
//											if (thisCount == null){
//												levelMapCount.put(i+1, 1);
//											}else{
//												levelMapCount.put(i+1, thisCount + 1);
//											}
//										}
//									}
//								}
								
								//2. deal with token
//								System.out.println("---" + thisStr);
								if (thisStr.trim().length() > 0){
									String [] tokenArray = thisStr.split(" ");
									for (int k=0; k<max_n; ++k){
										for (int l=k; l<tokenArray.length; ++l){
//											String thisToken = "";
											String thisStem = "";
											for (int m=l-k; m<=l; ++m){
												if (m != l-k){
													thisStem += " ";
												}
												String thisToken = tokenArray[m].trim().toLowerCase();
												thisStem += WordStemmer.getStem(thisToken);
											}
											HashMap<String, Integer> thisUUIDMapCount = stemMapUUID.get(thisStem);
											if (thisUUIDMapCount == null){
												thisUUIDMapCount = new HashMap<String, Integer>();
												stemMapUUID.put(thisStem, thisUUIDMapCount);
											}
											Integer thisCount = thisUUIDMapCount.get(thisUUID);
											if (thisCount == null){
												thisUUIDMapCount.put(thisUUID, 1);
											}else{
												thisUUIDMapCount.put(thisUUID, thisCount + 1);
											}
											thisCount = thisStemMapCount.get(thisStem);
											if (thisCount == null){
												thisStemMapCount.put(thisStem, 1);
											}else{
												thisStemMapCount.put(thisStem, thisCount + 1);
											}
											HashSet<Integer> thisSet = thisStemMapLevel.get(thisStem);
											if (thisSet == null){
												thisSet = new HashSet<Integer>();
												thisStemMapLevel.put(thisStem, thisSet);
											}
											thisSet.add(i+1);
										}
									}
								}
							}
						}
					}
					
					oneLine = br.readLine();
				}
			}catch (Exception e) {
				e.printStackTrace();
			}
			
			
			br.close();
			isr.close();
			fis.close();
		} catch (FileNotFoundException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}
}
